package engine

import (
	"crawlerm/fetcher"
	"log"
)

type SimpleEngine struct {

}
func (e SimpleEngine) Run(seeds ...Request){
	var requests []Request
	for _,r:=range seeds{
		requests=append(requests,r)
	}

	for len(requests)>0{
		r:=requests[0]
		requests=requests[1:]

		parseResult,err:=worker(r)
		if err!=nil{
			continue
		}
		requests=append(requests,parseResult.Requests...)

		for _,item:=range parseResult.Items{
			log.Printf("获取到item: %v",item)
		}
	}
}

func worker(r Request)(ParseResult,error){
	log.Printf("fetching: %s",r.Url)
	body,err:=fetcher.Fetch(r.Url)
	if err!=nil{
		log.Printf("错误的url%s和错误状态码 %s, %v",r.Url,err)
		return ParseResult{},err
	}
	return r.ParserFunc(body),nil
}
